/*
 Copyright (c) 2009, Ivan Popivanov
 All rights reserved.

 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions
 are met:

 * Redistributions of source code must retain the above copyright
 notice, this list of conditions and the following disclaimer.

 * Redistributions in binary form must reproduce the above copyright
 notice, this list of conditions and the following disclaimer in the
 documentation and/or other materials provided with the distribution.

 * Neither the name of StockCandles nor the names of its contributors
 may be used to endorse or promote products derived from this software
 without specific prior written permission.

 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
 TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <float.h>

#include <QVarLengthArray>
#include <QFileDialog>
#include <QProgressDialog>
#include <QTemporaryFile>
#include <QTextStream>
#include <QTcpSocket>
#include <QSqlQuery>
#include <QSqlError>
#include <QEventLoop>
#include <QTimer>
#include <QTime>
#include <QHttp>
#include <QHash>
#include <QFile>
#include <QUrl>
#include <QDir>

#include "ta_libc.h"

#include "Csv.h"
#include "AppEngn.h"
#include "Settings.h"
#include "Ohlc.h"

void
AppEngn::initGUI()
{
   mainWnd = new MainWindow();
   chartView = new ChartView(mainWnd);
   mainWnd->setCentralWidget(chartView);
   connect(mainWnd->actionAdd_EODData, SIGNAL(triggered()), SLOT(onAddEODData()));
   connect(mainWnd->actionClose, SIGNAL(triggered()), SLOT(onClose()));
}

void
AppEngn::startGUI()
{
   Q_ASSERT(mainWnd != NULL);
   mainWnd->show();
}

AppEngn::AppEngn()
       : mainWnd(NULL), chartView(NULL)
{
   logFile.setFileName("stockcandles.log");
   logFile.open(QFile::WriteOnly | QFile::Truncate);

   // db = QSqlDatabase::addDatabase("QSQLITE");
   // db.setDatabaseName(Settings::instance()->getDbPath());
}

static inline void
randInit(uint seed)
{
   srand(seed);
}

static inline void
randInit()
{
   QDateTime current = QDateTime::currentDateTime();
   randInit(current.toTime_t()*1000 + current.time().msec());
}

// A random double in [a..b]
static inline double
randDbl(double a, double b)
{
   return a + (b-a)*rand()/(RAND_MAX + 1.0);
}

static inline double
randDbl()
{
   return randDbl(0, 1);
}

static inline double
toPct(double gain)
{
   return (gain-1)*100;
}

QString
AppEngn::computeReturn(const QString &endDate, const QString &startDate, double factor, double maxErr, bool showDetails /* = false*/)
{
   if(endDate <= startDate)
   {
      return QString("The end date must be at least a day after the start date.");
   }

   QDate start = QDate::fromString(startDate, "yyyy-MM-dd");
   QDate end = QDate::fromString(endDate, "yyyy-MM-dd");

   // Find the indexes of the two dates
   int startId = ohlc.latest();
   int endId = ohlc.latest();
   while(startId >= 0)
   {
      if(end < ohlc.d[startId]) --endId;

      if(start < ohlc.d[startId]) --startId;
      else break;
   }

   if(!ohlc.isIdValid(startId)) return QString("Bad dates!");

   if(startId == endId) return QString("Not enough data between these two dates");

   randInit();

   QString allDetails;

   // Move from the start to the end and do the math
   double gain = 1.0;
   double maxGain = DBL_MIN;
   double minGain = DBL_MAX;
   for(int i = startId+1; i <= endId; ++i)
   {
      double dayGain = (ohlc.c[i] - ohlc.c[i+1])/ohlc.c[i+1];
      double dayErr = randDbl(-maxErr, +maxErr);
      gain *= 1.0 + dayGain*factor + dayErr;

      if(gain > maxGain) maxGain = gain;
      if(gain < minGain) minGain = gain;

      QString details(ohlc.d[i].toString("yyyy-MM-dd"));

      details += QString(": gain = %1 dayGain = %2 dayErr = %3").arg(toPct(gain), 2, 'f', 2).arg(dayGain, 2, 'f', 2).arg(dayErr, 2, 'f', 2);

      // Append a comma unless this is the last record
      if(allDetails.length() > 0)
      {
         details.append(", ");
      }

      allDetails = details + allDetails;
   }

   gain = toPct(gain);
   minGain = toPct(minGain);
   maxGain = toPct(maxGain);

   QString result = QString("gain = %1% maxGain = %2% minGain = %3%").arg(gain, 2, 'f', 2).arg(maxGain, 2, 'f', 2).arg(minGain, 2, 'f', 2);

   if(showDetails)
   {
      result += "\nDetails: " + allDetails;
   }
   return result;
}

bool
AppEngn::download(const QString &symbol, const QDate &startDate, const QString &path)
{
   QByteArray url = "http://ichart.finance.yahoo.com/table.csv?s=" + symbol.toUpper().toAscii();

   url += "&a=" + QString("%1").arg(startDate.month() - 1, 2, 10, QLatin1Char('0'));
   url += "&b=" + QString::number(startDate.day());
   url += "&c=" + startDate.toString("yyyy");

   QDate endDate = QDate::currentDate();

   url += "&d=" + QString("%1").arg(endDate.month() - 1, 2, 10, QLatin1Char('0'));
   url += "&e=" + QString::number(endDate.day());
   url += "&f=" + endDate.toString("yyyy");
   
   url += "&g=d&ignore=.csv";

   QHttp http("ichart.finance.yahoo.com");
   QEventLoop el;
   QTimer t;
   QFile file(path);

   if(!file.open(QFile::WriteOnly | QFile::Truncate)) { return false; }

   t.setSingleShot(true);

   QObject::connect(&http, SIGNAL(done(bool)), &el, SLOT(quit()));
   QObject::connect(&t, SIGNAL(timeout()), &el, SLOT(quit()));

   http.get(url, &file);   

   t.start(5000);

   el.exec();

   if(t.isActive())
   {
      t.stop();
   }
   else
   {
      return false;
   }

   return true;
}

bool
AppEngn::downloadNew(const QString &symbol, const QString &path)
{
   // Download to a temp location
   QTemporaryFile temp;
   if(!temp.open()) return false;
   if(!download(symbol, temp.fileName())) return false;
   if(!ohlc.loadFromYahooCsv(temp.fileName())) return false;

   // Copy to the path specified
   return temp.copy(path);
}

bool
AppEngn::updateFromYahoo(Ohlc &o, const QString &path)
{
   if(!onlyHolidaysSince(o.d[o.last()]))
   {
      Q_ASSERT(o.size() >= 7);
      // We need few values (using 7 currently) to compare the new data with the old
      QDate startDate = o.d[o.latest()].addDays(-7);

      QTemporaryFile temp;
      if(temp.open() && download(o.s, startDate, temp.fileName()))
      {
         Ohlc newOhlc;
         if(newOhlc.loadFromYahooCsv(temp.fileName()))
         {
            // Compare the common dates, if they are the same, then we
            // can just append the result. Otheriwse, we need to download
            // the complete ohlc.
            int id = newOhlc.dateToIndex(o.d[o.latest()]);
            if(id != -1)
            {
               bool different = false;
               for(int i = o.latest(), j = id; j >= 0; --i, --j)
               {
                  if(newOhlc.c[j] != o.c[i])
                  {
                     different = true;
                     break;
                  }
               }

               if(different)
               {
                  // Download the whole ohlc
                  if(!downloadNew(o.s, path)) return false;
                  if(!ohlc.loadFromYahooCsv(path)) return false;
               }
               else
               {
                  // Exclude today
                  ++id;

                  if(id <= newOhlc.latest())
                  {
                     // Update the ohlc both in memory and on disk
                     while(id <= newOhlc.latest())
                     {
                        ohlc.append(newOhlc.d[id], newOhlc.o[id], newOhlc.h[id], newOhlc.l[id], newOhlc.c[id], newOhlc.v[id]);
                        ++id;
                     }

                     // Write the series to disk
                     ohlc.storeToYahooCsv(path);
                  }
               }
            }
         }
         else return false;
      }
      else return false;
   }
   return true;
}

QString
AppEngn::loadSymbol(const QString &symbol, bool update /* = false */)
{
   ohlc.init(symbol);

   QString path = Settings::instance()->getDataDir() + "/" + symbol.toLower() + ".csv";
   if(!QFile::exists(path) || !ohlc.loadFromYahooCsv(path) || ohlc.size() < 7)
   {
      return downloadNew(symbol, path) ? "ok" : "failed";
   }
   else
   {
      Q_ASSERT(ohlc.size() >= 7);
      // Try to download newer data if an update was requested
      if(update)
      {
         return updateFromYahoo(ohlc, path) ? "ok" : "failed";
      }
   }

   return "ok";
}

QString
AppEngn::loadFromYahooCsv(const QString &symbol)
{
   QString path = Settings::instance()->getDataDir() + "/" + symbol.toLower() + ".csv";
   if(!QFile::exists(path))
   {
      // Download
      if(!downloadNew(symbol, path)) return "failed";
   }

   if(!ohlc.loadFromYahooCsv(path)) return "failed";
   chartView->setOhlc(ohlc);

   return "ok";
}

QString
AppEngn::listTAFunctions()
{
   QString result;
   QTextStream s(&result, QIODevice::WriteOnly);

   TA_StringTable *groups;
   TA_RetCode rc;

   rc = TA_GroupTableAlloc(&groups);

   if(rc == TA_SUCCESS)
   {
      TA_StringTable *functions;

      for(uint group = 0; group < groups->size; ++group)
      {
         rc = TA_FuncTableAlloc(groups->string[group], &functions);

         if(rc == TA_SUCCESS)
         {
            for(uint i = 0; i < functions->size; ++i)
            {
               const TA_FuncHandle *handle;
               const TA_FuncInfo *info;

               rc = TA_GetFuncHandle(functions->string[i], &handle);
               if(rc == TA_SUCCESS)
               {
                  rc = TA_GetFuncInfo(handle, &info);
                  if(rc == TA_SUCCESS)
                  {
                     s << info->group << ": " << info->name << endl;
                  }
               }
            }

            TA_FuncTableFree(functions);
         }
      }

      TA_GroupTableFree(groups);
   }

   return result;
}

bool
AppEngn::onlyHolidaysSince(const QDate &startDate)
{
   QDate now = QDate::currentDate().addDays(-1);   // Yesterday

   while(now > startDate)
   {
      if(now.dayOfWeek() != Qt::Saturday && now.dayOfWeek() != Qt::Sunday)
      {
         return false;
      }

      now = now.addDays(-1);
   }

   return true;
}

QStringList
AppEngn::stockListFromDir(const QString &path)
{
   QStringList list;

   QDir dir(path);
   if(!dir.exists()) return list;

   dir.setFilter(QDir::Files | QDir::NoSymLinks);
   QFileInfoList fileList = dir.entryInfoList();
   for(int i = 0; i < fileList.size(); ++i)
   {
      QString symbol = fileList.at(i).fileName();
      if(symbol.endsWith(".csv", Qt::CaseInsensitive))
      {
         symbol.chop(4);
         list.append(symbol);
      }
   }

   return list;
}

QString
AppEngn::updateStockList(const QStringList &stocks, const QString &outDir)
{
   // The symbols we weren't able to update
   QStringList failed;

   // Initialize the progress bar
   QProgressDialog progress("Updating Db", "Abort", 0, stocks.size(), mainWnd);
   progress.setWindowModality(Qt::WindowModal);
   progress.setValue(0);

   QStringListIterator iter(stocks);
   int pos = 0;
   while(iter.hasNext())
   {
      QString symbol = iter.next();
      ohlc.init(symbol);

      bool success = true;

      // Update the GUI
      progress.setValue(pos);
      progress.setLabelText(symbol.toUpper());

      if(progress.wasCanceled()) break;

      QString filePath = outDir + "/" + symbol.toLower() + ".csv";
      if(!QFile::exists(filePath) || !ohlc.loadFromYahooCsv(filePath) || ohlc.size() < 7)
      {
         if(!downloadNew(symbol, filePath)) success = false;
      }
      else
      {
         if(!updateFromYahoo(ohlc, filePath)) success = false;
      }

      if(!success)
      {
         failed.push_back(symbol);
      }

      ++pos;
   }
   progress.setValue(stocks.size());

   return "ok";
}

QString
AppEngn::updateStocks(const QString &pathIn)
{
   QString path;
   if(pathIn.isNull()) path = Settings::instance()->getDataDir();
   else path = pathIn;

   QStringList stocks = stockListFromDir(path);
   return updateStockList(stocks, path);
}

// Aligns the result from a TA function with the original series
//
// In the candle series the most recent candle is at position 0.
// The TA functions expect the most recent values to be at the
// end of the arrays. The result of the TA functions, starts at
// index zero and ends somewhere in the array. It ends with the
// most recent element. This function reverses the result so that
// the most recent element is at index 0, etc
template <class T>
static inline void
alignTAResult(T *t, int num)
{
   int steps = num / 2;
   T *t1 = t + (num - 1);
   T *t2 = t;
   for(int i=0; i<steps; ++i, ++t2, --t1)
   {
      T ttt = *t1;
      *t1 = *t2;
      *t2 = ttt;
   }
}

QString
AppEngn::computeStochastics(const QString &symbol, const QString &outPath, int kParam, int dParam, int smoothParam)
{
   QString result = "ok";

   CsvFile out(outPath);

   int outStart, outNum;

   TA_RetCode rc;

   Ohlc other;

   // Load the symbol series
   if(loadSymbol(symbol, true) != "ok") return "failed";
   qDebug("%d %d %d %d", ohlc.size(), ohlc.h.size(), ohlc.l.size(), ohlc.c.size());

   // The arrays for the stochastics
   DoubleArray kDaily(ohlc.size());
   DoubleArray dDaily(ohlc.size());

   int dailyFirst = INT_MIN;
   int weeklyFirst = INT_MIN;
   int monthlyFirst = INT_MIN;
   int quarterlyFirst = INT_MIN;

   // Compute the daily stochastics
   rc = TA_STOCH(0, ohlc.size()-1, ohlc.h.data(), ohlc.l.data(), ohlc.c.data(),
            kParam, dParam, TA_MAType_SMA, smoothParam, TA_MAType_SMA, &outStart, &outNum, kDaily.data(), dDaily.data());
   if(rc != TA_SUCCESS) return "failed";
   kDaily.resize(outNum); dDaily.resize(outNum);
   dailyFirst = outStart;

   // Compute the weekly stochastics
   other.fromDaily(ohlc, TimeFrame::weekly);

   // How many values we need for one stochastic
   int maxVals = kParam + dParam + smoothParam;
   DoubleArray kOut(maxVals);
   DoubleArray dOut(maxVals);
   int first = other.latest();
   DoubleArray kWeekly(other.size());
   DoubleArray dWeekly(other.size());
   Ohlc sample;
   while(sample.sample(other, first, TimeFrame::weekly, maxVals) == maxVals)
   {
      rc = TA_STOCH(0, maxVals-1, sample.h.data(), sample.l.data(), sample.c.data(),
               kParam, dParam, TA_MAType_SMA, smoothParam, TA_MAType_SMA, &outStart, &outNum, kOut.data(), dOut.data());
      if(rc != TA_SUCCESS) return "failed";
      kWeekly[first] = kOut[outNum - 1];
      dWeekly[first] = dOut[outNum - 1];
      --first;
   }
   weeklyFirst = first;

   // Compute the monthly stochastics
   other.fromDaily(ohlc, TimeFrame::monthly);

   // How many values we need for one stochastic
   first = other.latest();
   DoubleArray kMonthly(other.size());
   DoubleArray dMonthly(other.size());
   while(sample.sample(other, first, TimeFrame::monthly, maxVals) == maxVals)
   {
      rc = TA_STOCH(0, maxVals-1, sample.h.data(), sample.l.data(), sample.c.data(),
               kParam, dParam, TA_MAType_SMA, smoothParam, TA_MAType_SMA, &outStart, &outNum, kOut.data(), dOut.data());
      if(rc != TA_SUCCESS) return "failed";
      kMonthly[first] = kOut[outNum - 1];
      dMonthly[first] = dOut[outNum - 1];
      --first;
   }
   monthlyFirst = first;

   // Compute the quarterly stochastics
   other.fromDaily(ohlc, TimeFrame::quarterly);

   // How many values we need for one stochastic
   first = other.latest();
   DoubleArray kQuarterly(other.size());
   DoubleArray dQuarterly(other.size());
   while(sample.sample(other, first, TimeFrame::quarterly, maxVals) == maxVals)
   {
      rc = TA_STOCH(0, maxVals-1, sample.h.data(), sample.l.data(), sample.c.data(),
               kParam, dParam, TA_MAType_SMA, smoothParam, TA_MAType_SMA, &outStart, &outNum, kOut.data(), dOut.data());
      if(rc != TA_SUCCESS) return "failed";
      kQuarterly[first] = kOut[outNum - 1];
      dQuarterly[first] = dOut[outNum - 1];
      --first;
   }
   quarterlyFirst = first;

   // Output the results
   if(out.open(QFile::WriteOnly | QFile::Truncate))
   {
      if(!out.writeLine("Date,Open,High,Low,Close,Volume,DailyK,DailyD,WeeklyK,WeeklyD,MonthlyK,MonthlyD,QuarterlyK,QuarterlyD")) return "failed";
      for(int i = ohlc.latest(), j = 0; i >= ohlc.first(); --i, ++j)
      {
         QStringList fields;
         fields << ohlc.d[i].toString("yyyy-MM-dd") <<
                   QString::number(ohlc.o[i], 'f', 2) <<
                   QString::number(ohlc.h[i], 'f', 2) <<
                   QString::number(ohlc.l[i], 'f', 2) <<
                   QString::number(ohlc.c[i], 'f', 2) <<
                   QString::number(ohlc.v[i]);

         int id = kDaily.size() - 1 - j;
         if(id > dailyFirst)
         {
            fields << QString::number(kDaily[id], 'f', 2) << QString::number(dDaily[id], 'f', 2);
         }

         id = kWeekly.size() - 1 - j;
         if(id > weeklyFirst)
         {
            fields << QString::number(kWeekly[id], 'f', 2) << QString::number(dWeekly[id], 'f', 2);
         }

         id = kMonthly.size() - 1 - j;
         if(id > monthlyFirst)
         {
            fields << QString::number(kMonthly[id], 'f', 2) << QString::number(dMonthly[id], 'f', 2);
         }

         id = kQuarterly.size() - 1 - j;
         if(id > quarterlyFirst)
         {
            fields << QString::number(kQuarterly[id], 'f', 2) << QString::number(dQuarterly[id], 'f', 2);
         }

         if(!out.writeLine(fields)) return "failed";
      }
   }

   return result;
}

// engn.computeMA("f:/ttt/spx/spx.csv", "f:/ttt/spx")
QString
AppEngn::computeMA(const QString &inPath, const QString &outDir)
{
   int maLen = 12;
   QString result = "ok";

   // Load the series
   Ohlc ohlc;
   if(!ohlc.loadFromYahooCsv(inPath)) return "failed";

   // Open the output file
   CsvFile outSignals(outDir + "/signals.csv");
   if(!outSignals.open(QFile::WriteOnly | QFile::Truncate)) return "failed";
   QStringList fields;
   fields << "Date" << "Close" << "Signal";
   outSignals.writeLine(fields);

   TA_RetCode rc;

   // Compute the monthly series
   Ohlc other;
   other.fromDaily(ohlc, TimeFrame::monthly);
   for(int i = 0; i < other.size(); ++i)
   {
      qDebug() << other.d[i].toString("yyyy-MM-dd") << ", " << other.c[i];
   }

   DoubleArray dOut(ohlc.size());
   int outStart, outNum;
   int first = other.latest();
   Ohlc sample;
   int vals;
   while((vals = sample.sample(other, first, TimeFrame::monthly, other.size())) > 3)
   {
      qDebug() << other.d[other.latest()].toString("yyyy-MM-dd");
      double *pd = sample.c.data()+(sample.size()-vals);
      for(int i = 0; i < vals; ++i)
      {
         qDebug() << pd[i];
      }

      rc = TA_MA(0, vals-1, pd, maLen, TA_MAType_SMA, &outStart, &outNum, dOut.data());
      if(rc != TA_SUCCESS) return "failed";

      for(int i = 0; i < outNum; ++i)
      {
         qDebug() << dOut[i];
      }

      if(outNum < 3)
      {
         // Done
         break;
      }

      // Dump the MA for this date
      QString outFile = outDir + "/" + other.d[first].toString("yyyy-MM-dd") + ".csv";
      CsvFile outCsv(outFile);
      if(!outCsv.open(QFile::WriteOnly | QFile::Truncate)) return "failed";

      // Write the header
      fields.erase(fields.begin(), fields.end());
      fields << "Date" << "Close" << "MA";
      outCsv.writeLine(fields);

      // Write the data
      for(int i = outNum - 1, j = sample.latest(); i >= 0; --i, --j)
      {
         fields.erase(fields.begin(), fields.end());
         fields << sample.d[j].toString("yyyy-MM-dd") <<
                   QString::number(sample.c[j], 'f', 2) <<
                   QString::number(dOut[i], 'f', 2);
         outCsv.writeLine(fields);
      }

      qDebug() << sample.d[sample.latest()].toString("yyyy-MM-dd") <<
                  QString::number(sample.c[sample.latest()], 'f', 2) <<
                  QString::number(dOut[outNum-1], 'f', 2);

      qDebug() << sample.d[sample.latest()-1].toString("yyyy-MM-dd") <<
                  QString::number(sample.c[sample.latest()-1], 'f', 2) <<
                  QString::number(dOut[outNum-2], 'f', 2);

      if(dOut[outNum-1] <= sample.c[sample.latest()])
      {
         fields.erase(fields.begin(), fields.end());
         fields << sample.d[sample.latest()].toString("yyyy-MM-dd") <<
                   QString::number(sample.c[sample.latest()], 'f', 2) <<
                   QString("BULL");
         outSignals.writeLine(fields);
      }
      else
      {
         fields.erase(fields.begin(), fields.end());
         fields << sample.d[sample.latest()].toString("yyyy-MM-dd") <<
                   QString::number(sample.c[sample.latest()], 'f', 2) <<
                   "BEAR";
         outSignals.writeLine(fields);
      }

      --first;
   }

   return "ok";
}

QString
AppEngn::computeIndicators(const QString &symbol, const QString &outPath)
{
   QString result = "ok";

   CsvFile out(outPath);

   int outStart, outNum;

   TA_RetCode rc;

   Ohlc other;

   // Load the symbol series
   if(loadSymbol(symbol, true) != "ok") return "failed";

   // The arrays for the stochastics
   DoubleArray kDaily(ohlc.size());
   DoubleArray dDaily(ohlc.size());

   int dailyFirst = INT_MIN;
   int weeklyFirst = INT_MIN;
   int monthlyFirst = INT_MIN;
   int atrFirst = INT_MIN;

   int kParam = 14;
   int dParam = 3;
   int smoothParam = 3;

   // Compute the daily stochastics
   rc = TA_STOCH(0, ohlc.size()-1, ohlc.h.data(), ohlc.l.data(), ohlc.c.data(),
            kParam, dParam, TA_MAType_SMA, smoothParam, TA_MAType_SMA, &outStart, &outNum, kDaily.data(), dDaily.data());
   if(rc != TA_SUCCESS) return "failed";
   kDaily.resize(outNum); dDaily.resize(outNum);
   dailyFirst = outStart;

   // Compute the weekly stochastics
   other.fromDaily(ohlc, TimeFrame::weekly);

   // How many values we need for one stochastic
   int maxVals = kParam + dParam + smoothParam;
   DoubleArray kOut(maxVals);
   DoubleArray dOut(maxVals);
   int first = other.latest();
   DoubleArray kWeekly(other.size());
   DoubleArray dWeekly(other.size());
   Ohlc sample;
   while(sample.sample(other, first, TimeFrame::weekly, maxVals) == maxVals)
   {
      rc = TA_STOCH(0, maxVals-1, sample.h.data(), sample.l.data(), sample.c.data(),
               kParam, dParam, TA_MAType_SMA, smoothParam, TA_MAType_SMA, &outStart, &outNum, kOut.data(), dOut.data());
      if(rc != TA_SUCCESS) return "failed";
      kWeekly[first] = kOut[outNum - 1];
      dWeekly[first] = dOut[outNum - 1];
      --first;
   }
   weeklyFirst = first;

   // Compute the monthly stochastics
   other.fromDaily(ohlc, TimeFrame::monthly);

   // How many values we need for one stochastic
   first = other.latest();
   DoubleArray kMonthly(other.size());
   DoubleArray dMonthly(other.size());
   while(sample.sample(other, first, TimeFrame::monthly, maxVals) == maxVals)
   {
      rc = TA_STOCH(0, maxVals-1, sample.h.data(), sample.l.data(), sample.c.data(),
               kParam, dParam, TA_MAType_SMA, smoothParam, TA_MAType_SMA, &outStart, &outNum, kOut.data(), dOut.data());
      if(rc != TA_SUCCESS) return "failed";
      kMonthly[first] = kOut[outNum - 1];
      dMonthly[first] = dOut[outNum - 1];
      --first;
   }
   monthlyFirst = first;

   // Compute the daily ATR
   DoubleArray atr(ohlc.size());
   rc = TA_ATR(0, ohlc.size()-1, ohlc.h.data(), ohlc.l.data(), ohlc.c.data(), 11, &outStart, &outNum, atr.data());
   if(rc != TA_SUCCESS) return "failed";
   atr.resize(outNum);
   atrFirst = outStart;

   // Output the results
   if(out.open(QFile::WriteOnly | QFile::Truncate))
   {
      if(!out.writeLine("Date,Open,High,Low,Close,Volume,ATR,DailyK,DailyD,WeeklyK,WeeklyD,MonthlyK,MonthlyD,QuarterlyK,QuarterlyD")) return "failed";
      for(int i = ohlc.latest(), j = 0; i >= ohlc.first(); --i, ++j)
      {
         QStringList fields;
         fields << ohlc.d[i].toString("yyyy-MM-dd") <<
                   QString::number(ohlc.o[i], 'f', 2) <<
                   QString::number(ohlc.h[i], 'f', 2) <<
                   QString::number(ohlc.l[i], 'f', 2) <<
                   QString::number(ohlc.c[i], 'f', 2) <<
                   QString::number(ohlc.v[i]);

         if(i >= atrFirst)
         {
            fields << QString::number(atr[i-atrFirst], 'f', 2);
         }

         int id = kDaily.size() - 1 - j;
         if(id > dailyFirst)
         {
            fields << QString::number(kDaily[id], 'f', 2) << QString::number(dDaily[id], 'f', 2);
         }

         id = kWeekly.size() - 1 - j;
         if(id > weeklyFirst)
         {
            fields << QString::number(kWeekly[id], 'f', 2) << QString::number(dWeekly[id], 'f', 2);
         }

         id = kMonthly.size() - 1 - j;
         if(id > monthlyFirst)
         {
            fields << QString::number(kMonthly[id], 'f', 2) << QString::number(dMonthly[id], 'f', 2);
         }

         if(!out.writeLine(fields)) return "failed";
      }
   }

   return result;
}

QString
AppEngn::createDb(const QString &path)
{
   bool ok = db.open();
   QSqlQuery q;
   ok = q.exec("CREATE TABLE IF NOT EXISTS CANDLES(SYMBOL TEXT NOT NULL, "
                  "DATETIME TIMESTAMP NOT NULL, OPEN REAL, HIGH REAL, "
				  "LOW REAL, CLOSE REAL, VOLUME INTEGER)");
                  // "CONSTRAINT UNIQUE_CANDLE UNIQUE(SYMBOL,DATETIME))");
   db.close();

   if(ok) return "ok";
   else return "failed";
}

bool
AppEngn::addEODDataFileToDb(const QString &path)
{
   if(!QFile::exists(path)) return false;

   CsvFile file(path);
   if(!file.open(QFile::ReadOnly)) return false;

   // Skip the header line
   QStringList fields;
   if(!file.readLine(fields)) return false;

   // Prepare the select query
   QSqlQuery selQry(db);
   selQry.prepare("SELECT SYMBOL FROM CANDLES WHERE SYMBOL=? AND DATE(DATETIME)=?");

   // Prepare the insert query
   QSqlQuery fullInsQry(db);
   fullInsQry.prepare("INSERT OR REPLACE INTO CANDLES VALUES (?,?,?,?,?,?,?)");

   // Read the file and insert the data
   while(file.readLine(fields))
   {
      // Check the number of fields from the line
      if(fields.size() < 7) return false;

      bool ok = false;

      // Verify the data
      double open = fields.at(2).toDouble(&ok); if(!ok) return false;
      double high = fields.at(3).toDouble(&ok); if(!ok) return false;
      double low = fields.at(4).toDouble(&ok); if(!ok) return false;
      double close = fields.at(5).toDouble(&ok); if(!ok) return false;
      uint volume = fields.at(6).toDouble(&ok); if(!ok) return false;

      QDateTime dt = QDateTime::fromString(fields.at(1), "dd-MMM-yyyy");
      if(!dt.isValid())
      {
         dt = QDateTime::fromString(fields.at(1), "yyyyMMdd");
         if(!dt.isValid()) return false;
      }

      // Make sure the time component is zero-ed
      dt.time().setHMS(0, 0, 0);

      // Check whether the entry exists
      selQry.bindValue(0, fields.at(0));
      selQry.bindValue(1, dt.toString("yyyy-MM-dd"));
      if(!selQry.exec()) return false;

      qDebug() << fields.at(0) << dt.toString("yyyy-MM-dd");

      bool noResult = !selQry.next();
      selQry.finish();

      if(noResult)
      {
         // No entry for this candle - add entries
         QSqlQuery cntQry(db);
         if(!cntQry.exec("SELECT SYMBOL FROM CANDLES WHERE SYMBOL='" + fields.at(0) + "' LIMIT 1")) return false;

         QDateTime firstDate;
         firstDate.time().setHMS(0, 0, 0);

         QDateTime lastDate;
         lastDate.time().setHMS(0, 0, 0);

         noResult = !cntQry.next();
         cntQry.finish();

         if(noResult)
         {
            // The first insert for this symbol, add 10 years of history and 1 year of future
            QDateTime minDate = QDateTime::currentDateTime().addYears(-10);
            firstDate = qMin(minDate, dt);
            lastDate = QDateTime::currentDateTime().addYears(1);

            qDebug() << "\t first entry - adding history and future";
         }
         else
         {
            // There are records for the symbol - determine what dates need to be added
            QDateTime maxDate;
            QSqlQuery q(db);
            if(!q.exec("SELECT MAX(DATETIME) FROM CANDLES")) return false;
            q.finish();
            maxDate = q.value(0).toDateTime();

            if(dt > maxDate)
            {
               firstDate = dt;
               lastDate = firstDate.addYears(1);
            }
            else
            {
               if(!q.exec("SELECT MIN(DATETIME) FROM CANDLES")) return false;
               q.finish();

               QDateTime minDate = q.value(0).toDateTime();
               if(dt < minDate)
               {
                  lastDate = dt;
                  firstDate = lastDate.addYears(-1);
               }
               else
               {
                  // That's an error dt is in [minDate, maxDate] but there is no candle
                  return false;
               }
            }
         }

         QSqlQuery shortInsQry(db);
         shortInsQry.prepare("INSERT INTO CANDLES(SYMBOL,DATETIME) VALUES(?,?)");

         qDebug() << "\t inserting new entries: " << firstDate << ", " << lastDate;
         while(firstDate < lastDate)
         {
            int dayOfWeek = firstDate.date().dayOfWeek();
            if(dayOfWeek != Qt::Sunday && dayOfWeek != Qt::Saturday)
            {
               shortInsQry.bindValue(0, fields.at(0));
               shortInsQry.bindValue(1, firstDate);

               if(!shortInsQry.exec()) return false;
            }

            firstDate = firstDate.addDays(1);
         }
         shortInsQry.finish();
      }

      fullInsQry.bindValue(0, fields.at(0));
      fullInsQry.bindValue(1, dt);
      fullInsQry.bindValue(2, open);
      fullInsQry.bindValue(3, high);
      fullInsQry.bindValue(4, low);
      fullInsQry.bindValue(5, close);
      fullInsQry.bindValue(6, volume);

      if(!fullInsQry.exec()) return false;
      fullInsQry.finish();
   }

   return true;
}

void
AppEngn::onAddEODData()
{
   QString path = Settings::instance()->getEODDataLoadDir();
   if(path.isNull())
   {
      path = QDir::current().path();
   }
   QStringList selectedFiles = QFileDialog::getOpenFileNames(mainWnd, "File Open", path);
   if(selectedFiles.size() == 0) return;

   qDebug() << QFileInfo(selectedFiles.at(0)).path();
   Settings::instance()->setEODDataLoadDir(QFileInfo(selectedFiles.at(0)).path());

   if(!db.open()) return;

   QSqlQuery q;
   bool ok = q.exec("CREATE TABLE IF NOT EXISTS candles(SYMBOL TEXT NOT NULL, "
                        "DATETIME TIMESTAMP NOT NULL, OPEN REAL, HIGH REAL, "
						"LOW REAL, CLOSE REAL, VOLUME INTEGER)");
                        // "CONSTRAINT UNIQUE_CANDLE UNIQUE(SYMBOL,DATETIME))");
   if(!ok) {db.close(); return; }
   db.commit();

   const QCursor &savedCursor = mainWnd->cursor();
   mainWnd->setCursor(Qt::WaitCursor);

   QProgressDialog progress("Adding End of Day Data", "Abort", 0, selectedFiles.size(), mainWnd);
   progress.setWindowModality(Qt::WindowModal);
   progress.setValue(1);
   progress.show();

   int pos = 0;

   QStringListIterator iter(selectedFiles);
   while(iter.hasNext())
   {
      const QString &file = iter.next();

      // Update the GUI
      qApp->processEvents();
      if(progress.wasCanceled()) break;
      progress.setValue(pos);
      progress.setLabelText(file + " (" + QString::number(pos) + "/" + QString::number(selectedFiles.size()) + ")");
      qApp->processEvents();

      db.transaction();
      ok = addEODDataFileToDb(file);
      if(ok)
      {
         db.commit();
      }
      else
      {
         db.rollback();
         mainWnd->getConsole()->writeLine("Error: failed to load file " + file + " ...");
      }

      db.commit();

      ++pos;
   }

   progress.setValue(selectedFiles.size());
   mainWnd->setCursor(savedCursor);

   db.close();
}

static bool
stochasticTestPass(Ohlc &daily,int kParam, int dParam, int smoothParam, QString &out)
{
   int maxVals = kParam + dParam + smoothParam;

   int outStart, outNum;

   int latest = daily.d.size() - 1;

   // The stock must be at least 50% off it's one-year high
   double hi, low;
   if(!daily.findHighLow(daily.d[latest], TimeFrame::annual, hi, low)) return false;

   double change = (hi - daily.c[latest])/hi;

   if(change < 0.5) return false;

   // Daily stochastics
   DoubleArray kDaily(1);
   DoubleArray dDaily(1);

   TA_RetCode rc;

   rc = TA_STOCH(daily.size()-maxVals, daily.size()-1, daily.h.data(), daily.l.data(), daily.c.data(),
      kParam, dParam, TA_MAType_SMA, smoothParam, TA_MAType_SMA, &outStart, &outNum, kDaily.data(), dDaily.data());
   if(rc != TA_SUCCESS)
   {
      return false;
   }

   // Weekly Stochastics
   Ohlc w;
   w.fromDaily(daily, TimeFrame::weekly);
   DoubleArray kWeekly(1);
   DoubleArray dWeekly(1);

   Ohlc sample;
   if(sample.sample(w, w.size()-1, TimeFrame::weekly, maxVals) == maxVals)
   {
      rc = TA_STOCH(0, maxVals-1, sample.h.data(), sample.l.data(), sample.c.data(), kParam, dParam, TA_MAType_SMA,
               smoothParam, TA_MAType_SMA, &outStart, &outNum, kWeekly.data(), dWeekly.data());
      if(rc != TA_SUCCESS)
      {
         return false;
      }
   }
   else return false;

   // Monthly Stochastics
   Ohlc m;
   m.fromDaily(daily, TimeFrame::monthly);
   DoubleArray kMonthly(1);
   DoubleArray dMonthly(1);

   if(sample.sample(m, m.size()-1, TimeFrame::monthly, maxVals) == maxVals)
   {
      rc = TA_STOCH(0, maxVals-1, sample.h.data(), sample.l.data(), sample.c.data(), kParam, dParam, TA_MAType_SMA,
               smoothParam, TA_MAType_SMA, &outStart, &outNum, kMonthly.data(), dMonthly.data());
      if(rc != TA_SUCCESS)
      {
         return false;
      }
   }
   else return false;

   if((kDaily[0] < 20 && dDaily[0] < 20 && kDaily[0] > dDaily[0]) ||
      (kWeekly[0] < 20 && dWeekly[0] < 20 && kWeekly[0] > dWeekly[0]) ||
      (kMonthly[0] < 20 && dMonthly[0] < 20 && kMonthly[0] > dMonthly[0]))
   {
      return true;
   }

   return false;
}

QString
AppEngn::filter()
{
   if(!db.open()) return "failed";

   QSqlQuery q;

   // To avoid stocks that have been discontinued, we will only load
   // symbols that have data for the latest date in the database
   bool ok = q.exec("SELECT SYMBOL FROM CANDLES WHERE DATETIME = (SELECT MAX(DATETIME) FROM CANDLES)");
   if(!ok) {db.close(); return "failed";}

   QStringList symbols;
   while(q.next())
   {
      symbols.append(q.value(0).toString());
   }

   qDebug("%d symbols added", symbols.size());

   Ohlc ohlc;

   // Now cycle through all symbols
   QStringListIterator iter(symbols);
   while(iter.hasNext())
   {
      ok = q.exec("SELECT DATETIME,OPEN,HIGH,LOW,CLOSE,VOLUME FROM CANDLES WHERE SYMBOL = '" + iter.next() + "' ORDER BY DATETIME ASC");
      if(!ok) {db.close(); return "failed";}

      if(q.size() > 0)
      {
         ohlc.reserve(q.size());
         ohlc.resize(0);

         while(q.next())
         {
            qDebug() << q.value(0);
            if(!q.value(1).isNull())
            {
               ohlc.append(q.value(0).toDate(), q.value(1).toDouble(NULL), q.value(2).toDouble(NULL),
                  q.value(3).toDouble(NULL), q.value(4).toDouble(NULL), q.value(5).toUInt(NULL));
            }
         }
      }
   }

   db.close();
   return "ok";
}

void
AppEngn::onClose()
{
   mainWnd->close();
}

void
AppEngn::log(QtMsgType, const char *msg)
{
   logFile.write(msg);
   logFile.putChar('\n');
}

static inline double
pctChange(double d1, double d2)
{
   return (d2 - d1) / d2 * 100.0;
}

static const double DOWTHEORY_THRESHOLD = 3.0;

enum
{
   DOWTHEORY_NO_TREND,
   DOWTHEORY_UPTREND,
   DOWTHEORY_UPTREND_PULLBACK,
   DOWTHEORY_UPTREND_BOTTOM,
   DOWTHEORY_DOWNTREND,
   DOWTHEORY_DOWNTREND_PULLBACK,
   DOWTHEORY_DOWNTREND_TOP
};

QString
AppEngn::dowTheorySignals(const QString &path, const QString &outPath)
{
   if(!ohlc.loadFromYahooCsv(path)) return "failed";

   QStringList states;

   int state = DOWTHEORY_NO_TREND;
   int lastTop = -1;
   int lastBottom = -1;
   for(int i = 0; i < ohlc.size(); ++i)
   {
      qDebug() << ohlc.d[i].toString("yyyy-MM-dd");
      QString stateString;
      double change;
      switch(state)
      {
      case DOWTHEORY_NO_TREND:
         // No trend yet - change the state when the threshold is passed
         change = pctChange(ohlc.c[0], ohlc.c[i]);
         if(change > DOWTHEORY_THRESHOLD)
         {
            // Uptrend
            state = DOWTHEORY_UPTREND;
            lastTop = i;
         }
         else if(change < -DOWTHEORY_THRESHOLD)
         {
            // Downtrend
            state = DOWTHEORY_DOWNTREND;
            lastBottom = i;
         }
         stateString = "N/A";
         break;

      case DOWTHEORY_UPTREND:
         if(ohlc.c[i] > ohlc.c[lastTop])
         {
            // Update the top in an uptrend
            lastTop = i;
         }
         else if(pctChange(ohlc.c[lastTop], ohlc.c[i]) < -DOWTHEORY_THRESHOLD)
         {
            // We reached a greater than 3% pullback
            lastBottom = i;
            state = DOWTHEORY_UPTREND_PULLBACK;
         }
         stateString = "Bull";
         break;

      case DOWTHEORY_DOWNTREND:
         if(ohlc.c[i] < ohlc.c[lastBottom])
         {
            // Update the bottom in a downtrend
            lastBottom = i;
         }
         else if(pctChange(ohlc.c[lastBottom], ohlc.c[i]) > DOWTHEORY_THRESHOLD)
         {
            // We reached a greater than 3% pullback
            lastTop = i;
            state = DOWTHEORY_DOWNTREND_PULLBACK;
         }
         stateString = "Bear";
         break;

      case DOWTHEORY_UPTREND_PULLBACK:
         if(ohlc.c[i] > ohlc.c[lastTop])
         {
            // Trend confirmation
            state = DOWTHEORY_UPTREND;
         }
         else if(pctChange(ohlc.c[lastBottom], ohlc.c[i]) > DOWTHEORY_THRESHOLD)
         {
            // We reached a another turn up after the pullback, now wait
            // for either breaking the previous top, or the previous bottom.
            state = DOWTHEORY_UPTREND_BOTTOM;
         }
         else if(ohlc.c[i] < ohlc.c[lastBottom])
         {
            lastBottom = i;
         }
         stateString = "Bull";
         break;

      case DOWTHEORY_UPTREND_BOTTOM:
         if(ohlc.c[i] > ohlc.c[lastTop])
         {
            // Trend confirmation
            state = DOWTHEORY_UPTREND;
            lastTop = i;
         }
         else if(ohlc.c[i] < ohlc.c[lastBottom])
         {
            // Bear reversal
            state = DOWTHEORY_DOWNTREND;
            lastBottom = i;
         }
         stateString = "Bull";
         break;

      case DOWTHEORY_DOWNTREND_PULLBACK:
         if(ohlc.c[i] < ohlc.c[lastBottom])
         {
            // Trend confirmation
            state = DOWTHEORY_DOWNTREND;
         }
         else if(pctChange(ohlc.c[lastTop], ohlc.c[i]) < -DOWTHEORY_THRESHOLD)
         {
            // We reached another turn up after the pullback, now wait for 
            // either breaking the previous top, or the previous bottom.
            state = DOWTHEORY_DOWNTREND_TOP;
         }
         else if(ohlc.c[i] > ohlc.c[lastTop])
         {
            lastTop = i;
         }
         stateString = "Bear";
         break;

      case DOWTHEORY_DOWNTREND_TOP:
         if(ohlc.c[i] < ohlc.c[lastBottom])
         {
            // Trend confirmation
            state = DOWTHEORY_DOWNTREND;
            lastBottom = i;
         }
         else if(ohlc.c[i] > ohlc.c[lastTop])
         {
            // Bull reversal
            state = DOWTHEORY_UPTREND;
            lastTop = i;
         }
         stateString = "Bear";
         break;
      }

      states.append(stateString);
   }

   if(!outPath.isNull())
   {
      CsvFile csv(outPath);
      if(!csv.open(QFile::WriteOnly | QFile::Truncate))
      {
         return "failed";
      }

      for(int i = ohlc.last(); i >= 0; --i)
      {
         QStringList values;
         values << ohlc.d[i].toString("yyyy-MM-dd") <<
                   QString::number(ohlc.o[i], 'f', 2) <<
                   QString::number(ohlc.h[i], 'f', 2) <<
                   QString::number(ohlc.l[i], 'f', 2) <<
                   QString::number(ohlc.c[i], 'f', 2) <<
                   QString::number(ohlc.v[i]) <<
                   states[i];
         csv.writeLine(values);
      }
   }

   return "ok";
}

QString
AppEngn::refreshIndices(const QString &inPath, bool defaultList /* = false */)
{
   QString path;
   QStringList list;
   if(inPath.isNull())
   {
      path = Settings::instance()->getDataDir();
      list << "^DJI" << "^DJT" << "^DJU" << "^DJA" << "^DWC" << "^GSPC";
   }
   else
   {
      path = inPath;

      QDir dir(path);
      if(!dir.exists()) return "failed";

      if(!defaultList)
      {
         QStringList filters;
         filters << "^*.csv";
         QFileInfoList fileInfoList = dir.entryInfoList(filters, QDir::Files | QDir::NoSymLinks);
         for(int i = 0; i < fileInfoList.size(); ++i)
         {
            qDebug() << fileInfoList[i].completeBaseName();
            list.append(fileInfoList[i].completeBaseName());
         }
      }
      else
      {
         list << "^DJI" << "^DJT" << "^DJU" << "^DJA" << "^DWC" << "^GSPC";
      }
   }

   return updateStockList(list, path);
}

// Return the index of the nth element which is false
static inline uint
findNthFalse(const QVector<bool> &b, uint nth)
{
   int i;
   for(i = 0; i < b.size(); ++i)
   {
      if(!b[i])
      {
         if(nth == 0) break;
         else --nth;
      }
   }
   return i;
}

static inline void
generatePrices(double low, double high, double open, double close, QVector<double> &price, QVector<bool> &generated)
{
   Q_ASSERT(price.size() >= 4);
   Q_ASSERT(price.size() == generated.size());
   for(int i = 0; i < generated.size(); ++i)
   {
      generated[i] = false;
   }

   price[0] = open;
   generated[0] = true;

   price[price.size() - 1] = close;
   generated[generated.size() - 1] = true;

   uint left = generated.size() - 2;

   // Pick a point for the high
   uint id = randDbl(0, left - 1);
   price[id] = high;
   generated[id] = true;
   --left;

   // Pick a point for the low
   id = findNthFalse(generated, randDbl(0, left - 1));
   price[id] = low;
   generated[id] = true;
   --left;

   if(left == 0) return;

   // Generate up to 25% points at random
   uint endLoop = price.size() * 0.75;
   while(left > endLoop)
   {
      id = findNthFalse(generated, randDbl(0, left - 1));
      price[id] = randDbl(low, high);
      generated[id] = true;
      --left;
   }

   // Generate the rest of the points evenly distributed between the random segments
   int start = 0;
   while(start < (price.size() - 1))
   {
      if(generated[start + 1]) {++start; continue;}

      int end = start + 1;
      while(end < price.size() && !generated[end]) ++end;
      Q_ASSERT(end < price.size());

      double step = (price[end] - price[start])/(end - start);
      for(int i = start+1; i < end; ++i)
      {
         price[i] = price[start] + (i - start)*step;
      }

      start = end;
   }

   // Sanity checking
   /*
   bool seenMin = false;
   bool seenMax = false;
   for(int i = 0; i < price.size(); ++i)
   {
      Q_ASSERT(price[i] <= high && price[i] >= low);
      seenMin |= price[i] == low;
      seenMax |= price[i] == high;
   }
   Q_ASSERT(seenMin && seenMax);
   */
}

static inline bool
generateIntraday(const Ohlc &src, int updates, CsvFile &csv, CsvFile &updatesCsv, CsvFile &allCsv, CsvFile &allUpdatesCsv)
{
   // Generate the timestamps
   QList<QTime> times;

   QTime endTime(16, 0, 0);
   QTime currentTime(9, 0, 0);

   while(currentTime <= endTime)
   {
      times.append(currentTime);
      currentTime = currentTime.addSecs(60*10);
   }

   QStringList values;
   values << "Date" << "Price" << "Volume";
   csv.writeLine(values);
   updatesCsv.writeLine(values);

   // An array to hold the volumes for each tick of the day
   QVector<quint64> volumes;
   volumes.resize(times.size());

   // Arrays to generate the daily ticks
   QVector<double> prices;
   QVector<bool> generatedPrices;

   prices.resize(times.size());
   generatedPrices.resize(times.size());

   for(int i = 0; i < src.size(); ++i)
   {
      // Split the volume equally among all intervals
      quint64 v = src.v[i] / times.size();
      for(int j = 0; j < volumes.size(); ++j)
      {
         volumes[j] = v;
      }

      // Do some random shuffling to generate the volumes
      for(int j = 0; j < volumes.size(); ++j)
      {
         uint pos = randDbl(0, volumes.size()-2);
         if(pos >= j) pos += 1;
         uint diff = randDbl(0, volumes[j]);
         volumes[j] -= diff;
         volumes[pos] += diff;
      }

      // Generate the prices
      generatePrices(src.l[i], src.h[i], src.o[i], src.c[i], prices, generatedPrices);

      QDateTime dt(src.d[i]);
      for(int j = 0; j < times.size(); ++j)
      {
         dt.setTime(times[j]);

         values.clear();
         values << dt.toString("yyyy-MM-dd hh:mm:ss") <<
                   QString::number(prices[j], 'f', 2) <<
                   QString::number(volumes[j]);

         bool outputHistory = i < src.size() - updates;
         if(outputHistory)
         {
            csv.writeLine(values);
         }
         else
         {
            updatesCsv.writeLine(values);
         }

         values.clear();
         values << src.s.toUpper() <<
            dt.toString("yyyy-MM-dd hh:mm:ss") <<
                   QString::number(prices[j], 'f', 2) <<
                   QString::number(volumes[j]);
         if(outputHistory)
         {
            allCsv.writeLine(values);
         }
         else
         {
            allUpdatesCsv.writeLine(values);
         }

         dt = dt.addSecs(20*60);
      }
   }

   return true;
}

QString
AppEngn::generateData(const QString &outPath)
{
   if(!db.open()) return "failed";

   randInit();

   QSqlQuery q;

   // To avoid stocks that have been discontinued, we will only load
   // symbols that have data for the latest date in the database
   bool ok = q.exec("SELECT SYMBOL FROM CANDLES WHERE TIMESTAMP = (SELECT MAX(TIMESTAMP) FROM CANDLES)");
   if(!ok) {db.close(); return "failed";}

   QStringList symbols;
   while(q.next())
   {
      symbols.append(q.value(0).toString());
   }
   q.clear();

   Ohlc ohlc;

   CsvFile allCsv(outPath + "/alltogether.csv");
   if(!allCsv.open(QFile::WriteOnly | QFile::Truncate)) {db.close(); return "failed";}

   CsvFile allUpdatesCsv(outPath + "/alltogether.updates.csv");
   if(!allUpdatesCsv.open(QFile::WriteOnly | QFile::Truncate)) {db.close(); return "failed";}

   QDir dir(outPath + "/csv");
   if(!dir.exists())
   {
      dir.mkpath(outPath + "/csv");
      if(!dir.exists()) return "failed";
   }

   // Now cycle through all symbols
   QStringListIterator iter(symbols);
   QStringList intradaySymbols;
   while(iter.hasNext())
   {
      QString symbol = iter.next();
      ohlc.init(symbol);
      ohlc.resize(0);

      // Only use symbols with more than 1000 candles
      ok = q.exec("SELECT COUNT(SYMBOL) FROM CANDLES WHERE SYMBOL = '" + symbol + "'");
      if(!ok) {db.close(); return "failed";}
      q.next();
      uint resultSize = q.value(0).toInt(&ok);
      if(!ok) {db.close(); return "failed";}
      if(resultSize < 1000) continue;
      q.clear();

      ok = q.exec("SELECT TIMESTAMP,OPEN,HIGH,LOW,CLOSE,VOLUME FROM CANDLES WHERE SYMBOL = '" + symbol + "' ORDER BY TIMESTAMP ASC");
      if(!ok) {db.close(); return "failed";}

      ohlc.reserve(q.size());
      ohlc.resize(0);

      while(q.next())
      {
         if(!q.value(1).isNull())
         {
            ohlc.append(q.value(0).toDate(), q.value(1).toDouble(NULL), q.value(2).toDouble(NULL),
               q.value(3).toDouble(NULL), q.value(4).toDouble(NULL), q.value(5).toUInt(NULL));
         }
      }

      QString outCsvFile = outPath + "/" + symbol.toLower() + ".csv";

      CsvFile csv(outPath + "/csv/" + symbol.toLower() + ".csv");
      if(!csv.open(QFile::WriteOnly | QFile::Truncate)) {db.close(); return "failed";}

      CsvFile updatesCsv(outPath + "/csv/" + symbol.toLower() + ".updates.csv");
      if(!updatesCsv.open(QFile::WriteOnly | QFile::Truncate)) {db.close(); return "failed";}

      if(!generateIntraday(ohlc, 200, csv, updatesCsv, allCsv, allUpdatesCsv)) {db.close(); return "failed";}
      intradaySymbols.append(symbol);

      csv.close();
      updatesCsv.close();
   }

   QFile listFile(outPath + "/all.list");
   if(listFile.open(QFile::WriteOnly | QFile::Truncate))
   {
      for(int i = 0; i < intradaySymbols.size(); ++i)
      {
         listFile.write(intradaySymbols.at(i).toAscii() + "\n");
      }
   }

   db.close();
   return "ok";
}

QString
AppEngn::benchmarkCsvReads(const QString &path, int filesToProcess, int linesToProcess)
{
   QStringList symbols;
   QFile listFile(path + "/all.list");
   if(listFile.open(QFile::ReadOnly))
   {
      static char line[5000];
      int err;
      while(listFile.readLine(line, sizeof(line)-1) > 0)
      {
         symbols.append(QString(line).trimmed());
      }
   }
   else
   {
      return "failed (no all.list file in the path)";
   }

   if(symbols.size() <= 0) return "failed (all.list seems to be empty)";

   randInit();

   QTime t;
   t.start();
   for(int i = 0; i < filesToProcess; ++i)
   {
      int id = randDbl(0, symbols.size() - 1);
      QString filePath = path + "/csv/" + symbols.at(id).toLower() + ".csv";
      QFile csvFile(filePath);
      if(!csvFile.open(QFile::ReadOnly)) return QString("failed (couldn't open ") + filePath + ")";
      static char line[5000];
      qint64 err;
      int linesLeft = linesToProcess;
      while((err = csvFile.readLine(line, sizeof(line)-1)) > 0 && linesLeft > 0)
      {
         // Remove whitespaces at the end
         int last = err - 1;
         while(last >= 0 && (line[last] == '\n' || line[last] == '\r'))
         {
            line[last--] = '\0';
         }
         if(last == 0) return QString("failed (bad line in ") + filePath + ")";

         Csv csv;
         QStringList fields;
         if(!csv.parse(line, fields)) return QString("failed (bad line in ") + filePath + ")";
         --linesLeft;
      }
   }
   int totalTime = t.elapsed();
   double average = ((double)totalTime / filesToProcess) / 1000;

   return QString("ok (") + QString::number((double)totalTime/1000, 'f', 2) + " secs, " + QString::number(average, 'f', 4) + " average secs per file)";
}

QString
AppEngn::benchmarkCsvAppends(const QString &path, int appends)
{
   QFile csvUpdates(path + "/alltogether.updates.csv");
   if(!csvUpdates.open(QFile::ReadOnly)) {return "failed";}

   static char line[5000];

   int totalTime = 0;

   QTime t;
   t.start();
   for(int i = 0; i < appends; ++i)
   {
      qint64 err;
      err = csvUpdates.readLine(line, sizeof(line)-1);
      if(err <= 0) {return "failed (while reading from the updates file)";}

      // Remove whitespaces at the end
      int last = err - 1;
      while(last >= 0 && (line[last] == '\n' || line[last] == '\r'))
      {
         line[last--] = '\0';
      }
      if(last == 0) return QString("failed (bad line in the updates file");

      Csv csv;
      QStringList fields;
      if(!csv.parse(line, fields)) return QString("failed (while parsing the line)");

      QByteArray data;
      data.append(fields.at(1) + "," + fields.at(2) + "," + fields.at(3) + "\n");

      t.restart();
      QFile symbolFile(path + "/csv/" + fields.at(0).toLower() + ".csv");
      if(!symbolFile.open(QFile::Append | QFile::WriteOnly)) {return "failed (opening the file for writing)";}

      symbolFile.write(data);
      symbolFile.close();
      totalTime += t.elapsed();
   }

   double average = ((double)totalTime / appends) / 1000;

   return QString("ok (") + QString::number((double)totalTime/1000, 'f', 2) + " secs, " + QString::number(average, 'f', 4) + " average secs per file)";
}

QString
AppEngn::benchmarkFileIO(const QString &path, int filesToProcess)
{
   QStringList symbols;
   QFile listFile(path + "/all.list");
   if(listFile.open(QFile::ReadOnly))
   {
      static char line[5000];
      int err;
      while(listFile.readLine(line, sizeof(line)-1) > 0)
      {
         symbols.append(QString(line).trimmed());
      }
   }
   else
   {
      return "failed (no all.list file in the path)";
   }

   if(symbols.size() <= 0) return "failed (all.list seems to be empty)";

   randInit();

   int openTime = 0;
   int seekTime = 0;
   int closeTime = 0;

   QTime t;
   t.start();
   for(int i = 0; i < filesToProcess; ++i)
   {
      int id = randDbl(0, symbols.size() - 1);
      QString filePath = path + "/csv/" + symbols.at(id).toLower() + ".csv";
      QFile csvFile(filePath);
      t.restart();
      if(!csvFile.open(QFile::ReadOnly)) return QString("failed (couldn't open ") + filePath + ")";
      openTime += t.elapsed();

      t.restart();
      csvFile.seek(csvFile.size());
      seekTime += t.elapsed();

      t.restart();
      csvFile.close();
      closeTime += t.elapsed();
   }

   return QString("ok (") + QString::number((double)openTime/1000, 'f', 2) + " secs, " + QString::number((double)seekTime/1000, 'f', 2) + " secs, " + QString::number((double)closeTime/1000, 'f', 2) + " secs)";
}

QString
AppEngn::benchmarkDb(const QString &path, int symbolsToProcess, int linesToProcess)
{
   QSqlDatabase localDb = QSqlDatabase::addDatabase("QSQLITE");
   localDb.setDatabaseName(path + "/benchmark.sqlite");
   if(!localDb.open()) {qDebug() << localDb.lastError().text(); return "failed (error opening the database)";}

   // Read the symbols from a file because SELECT DISTINCT SYMBOL FROM CANDLES is
   // taking foever with SQLLITE
   QStringList symbols;
   QFile listFile(path + "/all.list");
   if(listFile.open(QFile::ReadOnly))
   {
      static char line[5000];
      int err;
      while(listFile.readLine(line, sizeof(line)-1) > 0)
      {
         symbols.append(QString(line).trimmed());
      }
   }
   else
   {
      return "failed (no all.list file in the path)";
   }

   if(symbols.size() <= 0) return "failed (all.list seems to be empty)";

   randInit();

   // Prepare the select query
   QSqlQuery q(localDb);
   bool ok;

   /*
   ok = q.exec("SELECT DISTINCT SYMBOL FROM CANDLES");
   
   if(!ok) {localDb.close(); return "failed";}

   QStringList symbols;
   while(q.next())
   {
      symbols.append(q.value(0).toString());
   }
   q.clear();

   if(symbols.size() <= 0) return "failed (no symbols in the database)";
   */

   QTime t;
   t.start();
   for(int i = 0; i < symbolsToProcess; ++i)
   {
      int id = randDbl(0, symbols.size() - 1);
      ok = q.exec("SELECT TIMESTAMP, PRICE FROM CANDLES WHERE SYMBOL = \'" + symbols.at(id) + "\'");
      if(!ok) {localDb.close(); return "failed (for symbol " + symbols.at(id) + ")";}
      int linesLeft = linesToProcess;
      while(q.next() && linesLeft > 0)
      {
         --linesLeft;
      }
      q.clear();
   }
   int totalTime = t.elapsed();
   double average = ((double)totalTime / symbolsToProcess) / 1000;

   localDb.close();
   return QString("ok (") + QString::number((double)totalTime/1000, 'f', 2) + " secs, " + QString::number(average, 'f', 4) + " average secs per time series)";
}

QString
AppEngn::benchmarkFastDbReads(const QString &path, int symbolsToProcess, int linesToProcess)
{
   QSqlDatabase localDb = QSqlDatabase::addDatabase("QSQLITE");
   localDb.setDatabaseName(path + "/fast.sqlite");
   if(!localDb.open()) {qDebug() << localDb.lastError().text(); return "failed (error opening the database)";}

   // Read the symbols from a file because SELECT DISTINCT SYMBOL FROM CANDLES is
   // taking foever with SQLLITE
   QStringList symbols;
   QFile listFile(path + "/all.list");
   if(listFile.open(QFile::ReadOnly))
   {
      static char line[5000];
      int err;
      while(listFile.readLine(line, sizeof(line)-1) > 0)
      {
         symbols.append(QString(line).trimmed());
      }
   }
   else
   {
      return "failed (no all.list file in the path)";
   }

   if(symbols.size() <= 0) return "failed (all.list seems to be empty)";

   randInit();

   // Prepare the select query
   QSqlQuery q(localDb);
   bool ok;

   /*
   ok = q.exec("SELECT DISTINCT SYMBOL FROM CANDLES");
   
   if(!ok) {localDb.close(); return "failed";}

   QStringList symbols;
   while(q.next())
   {
      symbols.append(q.value(0).toString());
   }
   q.clear();

   if(symbols.size() <= 0) return "failed (no symbols in the database)";
   */

   QHash<QString, qint64> symbolToRowid;

   // Load all symbols in memory
   ok = q.exec("SELECT SYMBOL, ROWID FROM INSTRUMENTS");
   if(!ok) {return "failed (reading the instruments table)";}
   while(q.next())
   {
      qint64 rowid = q.value(1).toLongLong(&ok);
      if(!ok) {return "failed (converting the rowid)";}
      symbolToRowid.insert(q.value(0).toString(), rowid);
   }
   q.clear();

   int totalTime = 0;

   QTime t;
   t.start();
   for(int i = 0; i < symbolsToProcess; ++i)
   {
      int id = randDbl(0, symbols.size() - 1);

      // Find the id for this symbol
      /*
      ok = q.exec("SELECT ROWID FROM INSTRUMENTS WHERE SYMBOL = '" + symbols.at(id) + "'");
      if(!ok) {return "failed (selecting from the instruments table)";}
      if(!q.next()) {return "failed (no record for the instrument)";}
      qint64 rowid = q.value(0).toLongLong(&ok);
      if(!ok) {return "failed (converting the rowid)";}
      q.clear();
      */

      t.restart();
      qint64 rowid = symbolToRowid[symbols.at(id)];

      qint64 rangeStart = (rowid << 32);
      qint64 rangeEnd = rangeStart | Q_INT64_C(0x00000000FFFFFFFF);

      ok = q.exec("SELECT PRICE, VOLUME FROM CANDLES WHERE KEY >= '" + QString::number(rangeStart) + "' AND KEY <= '" + QString::number(rangeEnd) + "'");
      if(!ok) {localDb.close(); return "failed (for symbol " + symbols.at(id) + ")";}
      int linesLeft = linesToProcess;
      while(q.next() && linesLeft > 0)
      {
         --linesLeft;
      }
      q.clear();
      totalTime += t.elapsed();
   }
   double average = ((double)totalTime / symbolsToProcess) / 1000;

   localDb.close();
   return QString("ok (") + QString::number((double)totalTime/1000, 'f', 2) + " secs, " + QString::number(average, 'f', 4) + " average secs per time series)";
}

QString
AppEngn::benchmarkFastDbUpdates(const QString &path, int updates)
{
   QSqlDatabase localDb = QSqlDatabase::addDatabase("QSQLITE");
   localDb.setDatabaseName(path + "/fast.sqlite");
   if(!localDb.open()) {qDebug() << localDb.lastError().text(); return "failed (error opening the database)";}

   // Read the symbols from a file because SELECT DISTINCT SYMBOL FROM CANDLES is
   // taking forever with SQLLITE
   QStringList symbols;
   QFile listFile(path + "/all.list");
   if(listFile.open(QFile::ReadOnly))
   {
      static char line[5000];
      int err;
      while(listFile.readLine(line, sizeof(line)-1) > 0)
      {
         symbols.append(QString(line).trimmed());
      }
   }
   else
   {
      return "failed (no all.list file in the path)";
   }

   if(symbols.size() <= 0) return "failed (all.list seems to be empty)";

   randInit();

   // Prepare the select query
   QSqlQuery q(localDb);
   bool ok;

   // Load all symbols in memory
   QHash<QString, qint64> symbolToRowid;
   ok = q.exec("SELECT SYMBOL, ROWID FROM INSTRUMENTS");
   if(!ok) {return "failed (reading the instruments table)";}
   while(q.next())
   {
      qint64 rowid = q.value(1).toLongLong(&ok);
      if(!ok) {return "failed (converting the rowid)";}
      symbolToRowid.insert(q.value(0).toString(), rowid);
   }
   q.clear();

   /*
   ok = q.exec("PRAGMA SYNCHRONOUS=OFF");
   if(!ok) {return "failed (settings synchronous to off)";}
   q.clear();
   */

   qint64 rowid = symbolToRowid[QString("GE")];

   // Load all timestamps for GE in memory - use them to randomly pick a timestamp
   QList<qint64> timestamps;
   qint64 rangeStart = (rowid << 32);
   qint64 rangeEnd = rangeStart | Q_INT64_C(0x00000000FFFFFFFF);

   ok = q.exec("SELECT ROWID FROM CANDLES WHERE KEY >= '" + QString::number(rangeStart) + "' AND KEY <= '" + QString::number(rangeEnd) + "'");
   if(!ok) {localDb.close(); return "failed (loading all GE timestamps)";}
   while(q.next())
   {
      qint64 timestamp = q.value(0).toLongLong(&ok);
      if(!ok) {return "failed";}
      timestamps.append(timestamp & Q_INT64_C(0x00000000FFFFFFFF));
   }
   q.clear();

   QString price = QString::number(53.81, 'f', 2);

   int failed = 0;

   int updatesBeforeCommit = 200;

   localDb.transaction();
   QTime t;
   t.start();
   int totalTime = 0;
   for(int i = 0; i < updates; ++i)
   {
      int id = randDbl(0, symbols.size() - 1);

      rowid = symbolToRowid[symbols.at(id)];

      // id = randDbl(0, timestamps.size() - 1);
      id = randDbl(0, 300);
      qint64 key = (rowid << 32) | timestamps.at(id);

      t.restart();
      ok = q.exec("UPDATE CANDLES SET PRICE = '" + price + "' WHERE KEY = '" + QString::number(key) + "'");
      if(!ok) {++failed;}
      q.clear();

      if(--updatesBeforeCommit == 0)
      {
         localDb.commit();
         localDb.transaction();
         updatesBeforeCommit = 200;
      }
      totalTime += t.elapsed();
   }

   localDb.commit();

   double average = ((double)totalTime / updates) / 1000;

   localDb.close();
   return QString("ok (") + QString::number((double)totalTime/1000, 'f', 2) + " secs, " + QString::number(average, 'f', 4) + " average secs per update, " +
            QString::number(failed) + " updates failed)";
}

QString
AppEngn::loadFastDb(const QString &path, uint numCandles /* = UINT_MAX */)
{
   QSqlDatabase localDb = QSqlDatabase::addDatabase("QSQLITE");
   localDb.setDatabaseName(path + "/new.sqlite");
   if(!localDb.open()) {qDebug() << localDb.lastError().text(); return "failed (error opening the database)";}

   // Create the table
   QSqlQuery q;

   bool ok = q.exec("CREATE TABLE IF NOT EXISTS CANDLES (KEY INTEGER PRIMARY KEY, "
				  "PRICE REAL, VOLUME INTEGER)");
   if(!ok) {qDebug() << localDb.lastError().text(); return "failed (error creating the CANDLES table)";}
   q.clear();

   ok = q.exec("DELETE FROM CANDLES");
   if(!ok) {qDebug() << localDb.lastError().text(); return "failed (deleting from CANDLES)";}
   q.clear();

   ok = q.exec("CREATE TABLE IF NOT EXISTS INSTRUMENTS (KEY INTEGER PRIMARY KEY AUTOINCREMENT, SYMBOL TEXT NOT NULL, EXCHANGE TEXT)");
   if(!ok) {qDebug() << localDb.lastError().text(); return "failed (error creating the SYMBOLS table)";}
   q.clear();

   ok = q.exec("DELETE FROM INSTRUMENTS");
   if(!ok) {qDebug() << localDb.lastError().text(); return "failed (deleting from INSTRUMENTS)";}
   q.clear();

   // Read the symbols from a file because SELECT DISTINCT SYMBOL FROM CANDLES is
   // taking foever with SQLLITE
   QStringList symbols;
   QFile listFile(path + "/all.list");
   if(listFile.open(QFile::ReadOnly))
   {
      static char line[5000];
      int err;
      while(listFile.readLine(line, sizeof(line)-1) > 0)
      {
         symbols.append(QString(line).trimmed());
      }
   }
   else
   {
      return "failed (no all.list file in the path)";
   }

   if(symbols.size() <= 0) return "failed (all.list seems to be empty)";

   for(int i = 0; i < symbols.size(); ++i)
   {
      ok = q.exec("INSERT INTO INSTRUMENTS (SYMBOL, EXCHANGE) VALUES ('" + symbols.at(i) + "', 'NYSE')");
      if(!ok) {qDebug() << localDb.lastError().text(); return "failed (inserting into the instruments table)";}
      q.clear();
   }

   ok = q.exec("CREATE UNIQUE INDEX IF NOT EXISTS INSTRUMENTID ON INSTRUMENTS (SYMBOL, EXCHANGE)");
   if(!ok) {qDebug() << localDb.lastError().text(); return "failed (creating the instruments table index)";}
   q.clear();

   QFile csvFile(path + "/alltogether.csv");
   if(!csvFile.open(QFile::ReadOnly)) {return "failed (opening alltogether.csv)";}
   
   static char line[5000];
   qint64 err;
   int linesToCommit = 100;
   db.transaction();
   while((err = csvFile.readLine(line, sizeof(line)-1)) > 0 && numCandles > 0)
   {
      // Remove whitespaces at the end
      int last = err - 1;
      while(last >= 0 && (line[last] == '\n' || line[last] == '\r'))
      {
         line[last--] = '\0';
      }
      if(last == 0) return QString("failed (bad line)");

      Csv csv;
      QStringList fields;
      if(!csv.parse(line, fields)) return QString("failed (bad line)");

      qDebug() << fields;

      // Find the id for this symbol
      ok = q.exec("SELECT ROWID FROM INSTRUMENTS WHERE SYMBOL = '" + fields.at(0) + "'");
      if(!ok) {return "failed (selecting from the instruments table)";}
      if(!q.next()) {return "failed (no record for the instrument)";}
      qint64 rowid = q.value(0).toLongLong(&ok);
      if(!ok) {return "failed (converting the rowid)";}
      q.clear();

      // Convert the time stamp
      QDateTime dt = QDateTime::fromString(fields.at(1), "yyyy-MM-dd hh:mm:ss");
      qint64 secs = dt.toTime_t();

      qint64 key = (rowid << 32) | secs;

      ok = q.exec("INSERT INTO CANDLES (KEY, PRICE, VOLUME) VALUES ('" + QString::number(key) + "', '" +
               fields.at(2) + "', '" + fields.at(3) + "')");
      if(!ok) {return "failed (inserting into candles)";}
      q.clear();

      if(linesToCommit == 0)
      {
         localDb.commit();
         localDb.transaction();
         linesToCommit = 100;
      }
      --linesToCommit;
      --numCandles;
   }

   return "ok";
}

AppEngn *AppEngn::appEngn = NULL;